from __future__ import print_function

import sys
try:
    sys.path.append("..")
    #reload(sys)
    sys.setdefaultencoding('utf-8')
except:
    pass

import datetime
import codecs
from textrank4zh import TextRank4Keyword
import requests
from bs4 import BeautifulSoup
import codecs

tr4w = TextRank4Keyword()

def text_keyword(text):
    tr4w.analyze(text=text, lower=True, window=3, pagerank_config={'alpha': 0.85})
    words = []
    for item in tr4w.get_keywords(5, word_min_len=2):
        # print(item.word )
        words.append(item.word)
    print(words)
    print('--抽取网页关键词--')
    phrases = []
    for phrase in tr4w.get_keyphrases(keywords_num=10, min_occur_num=1):
        phrases.append(phrase)
    print(phrases)
    return

def fun(tag):
    # 解析数据
    for tag in tags:
        print(tag.h2.get_text(strip=True))  # 打印一级标题
        #print(type(tag.p.get_text()))
        text=tag.p.get_text()
        text_keyword(text)
        print(tag.em.get_text(strip=True))
    return

text = codecs.open('term.txt', 'r', 'utf-8').read().split('\r\n')
# ['水泥', '回转窑', '熟料']
print(text)

wd='水泥'
for i in range(1,5):
    print(i)
    param = {'wd': wd, 'p': i
             }
    print(param)
    r = requests.get('https://www.dcement.com/search/s.aspx?c=&f=0&o=1',params=param,timeout=10)
    soup2 = BeautifulSoup(r.content,'lxml',from_encoding='utf-8')
    tags=soup2.find_all('li')
    print(len(tags))
    # if(len(tags)==0):
    #     break
    fun(tags)

#content=soup2('content')
    #soup2.select("#content")
#print(type(tags))


